Event logs studied:

logs = read.csv("../logs_summary.csv")
DT::datatable(logs, filter = "top")

Training configuration:

Surveyed techniques:

Baselines

filenames <- list.files()[grep(paste("^validation_Aalst_(?=.*\\.csv)", sep=''), list.files(), perl=TRUE)]
for (filename in filenames) {
  this = read.csv(filename, sep=",", header = TRUE)
  this = this[this$nr_events < 11,]
  #this = this[this$metric == "mae",]
  this$nr_cases = (this$nr_cases/sum(this$nr_cases))
  this$score_weighted = this$score * this$nr_cases
  this$method2 = this$cls
  this$method3 = ""
  this$filename = filename
  dat = rbind(dat, this)
}
filenames <- list.files()[grep(paste("^validation_SPN_(?=.*\\.csv)", sep=''), list.files(), perl=TRUE)]
for (filename in filenames) {
  this = read.csv(filename, sep=",", header = TRUE)
  this = this[this$nr_events < 11,]
  #this = this[this$metric == "mae",]
  this$nr_cases = this$nr_cases/sum(this$nr_cases)
  this$score_weighted = this$score * this$nr_cases
  this$method2 = this$cls
  this$method3 = ""
  this$filename = filename
  dat = rbind(dat, this)
}

Average MAE across all prefix lengths for all datasets and methods:

agg_MAE_unw = as.data.frame(aggregate(dat$score,by=list(dataset=dat$dataset, method=dat$method), mean))
agg_MAE_w = as.data.frame(aggregate(dat$score_weighted,by=list(dataset=dat$dataset, method=dat$method), sum))
agg_MAE_unw[,3] = round(agg_MAE_unw[,3], 3)
agg_MAE_w[,3] = round(agg_MAE_w[,3], 3)
colnames(agg_MAE_unw)[3] = "Avg_unweighted_MAE_days"
colnames(agg_MAE_w)[3] = "Avg_weighted_MAE_days"
agg_MAE = merge(agg_MAE_w, agg_MAE_unw)
agg_MAE = agg_MAE[with(agg_MAE, order(dataset, Avg_weighted_MAE_days, method = "radix")), ]
DT::datatable(agg_MAE, filter = "top")
tmp = ddply(dat, .(dataset,method), summarize, std=sd(score))
tmp$std = round(tmp$std,3)
agg_MAE = merge(agg_MAE, tmp)
agg_MAE$Avg_unweighted_MAE_days=NULL
agg_MAE = agg_MAE[with(agg_MAE, order(dataset, Avg_weighted_MAE_days, method = "radix")), ]
DT::datatable(agg_MAE, filter = "top")

Plots MAE vs prefix size

dat$method[dat$method=="_best_Aalst"] = "TS (best)"
dat$method[dat$method=="adaptive_FA_zero_combined"] = "adaptive FA"
dat$method[dat$method=="mean_FA_zero_combined"] = "mean FA"
dat$method[dat$method=="predictive_FA_prefix_index"] = "predictive FA (multiple)"
dat$method[dat$method=="predictive_FA_zero_combined"] = "predictive FA (single)"
dat$method[dat$method=="simple_regression_prefix_index"] = "black box (multiple)"
dat$method[dat$method=="simple_regression_zero_combined"] = "black box (single)"
dat$method[dat$method=="_SPN_Rogge-Solti"] = "stochastic Petri net"
levels(dat$dataset)[1] = "BPIC'12 A"
levels(dat$dataset)[2] = "BPIC'12 O"
levels(dat$dataset)[3] = "BPIC'12 W_n|1|"
levels(dat$dataset)[4] = "BPIC'12 W"
levels(dat$dataset)[5] = "CR"
levels(dat$dataset)[6] = "Helpdesk"
levels(dat$dataset)[7] = "Hospital"
levels(dat$dataset)[8] = "Invoice"
levels(dat$dataset)[9] = "RTFMP"
for (df in levels(dat$dataset)) {
  p = ggplot(dat[dat$dataset == df,], aes(x = nr_events, y = score, color = method)) + geom_point() + geom_line() + 
    #facet_wrap(~ dataset, ncol = 3, scales = "free") + 
    ylab("MAE, days") + xlab("Prefix length")+
    ggtitle(df)
    theme(legend.position="top")+
    theme(panel.background = element_rect(fill = 'white', colour = 'black',size=0.5)) + 
    theme(panel.grid.major = element_line(colour = 'lightgrey', size = 0.3))+
  theme(legend.background = element_rect(colour = 'white',size = 0.1, linetype='solid'))+
  theme(legend.title=element_blank())
  #print(p)
}

Interactive plots can be accessed here

for paper only

agg_MAE$method[agg_MAE$method=="_best_Aalst"] = "TS (best)"
agg_MAE$method[agg_MAE$method=="adaptive_FA_zero_combined"] = "adaptive FA"
agg_MAE$method[agg_MAE$method=="mean_FA_zero_combined"] = "mean FA"
agg_MAE$method[agg_MAE$method=="predictive_FA_prefix_index"] = "predictive FA (multiple)"
agg_MAE$method[agg_MAE$method=="predictive_FA_zero_combined"] = "predictive FA (single)"
agg_MAE$method[agg_MAE$method=="simple_regression_prefix_index"] = "black box (multiple)"
agg_MAE$method[agg_MAE$method=="simple_regression_zero_combined"] = "black box (single)"
agg_MAE$method[agg_MAE$method=="_SPN_Rogge-Solti"] = "stochastic Petri net"
dat = data.frame()
for (dset in levels(agg_MAE$dataset)) {
  df = agg_MAE[agg_MAE$dataset==dset,]
  df$Order = rank(df$Avg_weighted_MAE_days,ties.method = "min")
  dat = rbind(dat, df)
}
dat_ = ddply(dat, .(method), summarize, Mean=mean(Order))
foo = ddply(dat, .(method), summarize, std=sd(Order))
dat_ = merge(dat_,foo)
dat_ = dat_[order(dat_$Mean),]
pdf(file="mean-ranking.pdf",family="CM Roman",width=4,height=2)
p=ggplot(dat_, aes(x=reorder(method, -Mean), y=Mean, fill=method, group=method)) + 
  geom_bar(stat="identity") + theme(text=element_text(size=24))+
  geom_errorbar(aes(ymin=Mean-std, ymax=Mean+std),width=.2) + 
  theme_bw() + coord_flip() + theme(legend.position="none") + ylab("mean ranking")+
  theme(axis.title.y=element_blank(),axis.text=element_text(color="black"),axis.ticks.y=element_blank())+
  scale_y_continuous(expand = c(0,0), limits=c(0,9),breaks = seq(1,7,2), labels =as.character(seq(1,7,2)))+
  scale_fill_manual(values=cbPalette)
print(p)
dev.off()
null device 
          1 
embed_fonts("mean-ranking.pdf",outfile="mean-ranking.pdf")
ggplotly(p)

dat$case_duration = 0
dat$case_duration[dat$dataset=="BPI2012A"] = 7.5
dat$case_duration[dat$dataset=="BPI2012O"] = 15.1
dat$case_duration[dat$dataset=="BPI2012W"] = 11.4
dat$case_duration[dat$dataset=="BPI2012W_no_dup"] = 11.4
dat$case_duration[dat$dataset=="CreditRequirement"] = 0.95
dat$case_duration[dat$dataset=="helpdesk"] = 7.3
dat$case_duration[dat$dataset=="minit_invoice_10"] = 2.225
dat$case_duration[dat$dataset=="traffic_fines_139"] = 582
dat$case_duration[dat$dataset=="hospital_billing_977"] = 165.2
dat$Avg_weighted_MAE_days = dat$Avg_weighted_MAE_days / dat$case_duration
dat$std = dat$std / dat$case_duration
dat$Order = NULL
dat_ = ddply(dat, .(method), summarize, Mean=mean(Avg_weighted_MAE_days))
foo = ddply(dat, .(method), summarize, std=sd(Avg_weighted_MAE_days))
dat_ = merge(dat_,foo)
pdf(file="mean-ranking2.pdf",family="CM Roman",width=4,height=2)
p=ggplot(dat_, aes(x=reorder(method, -Mean), y=Mean, fill=method, group=method)) + 
  geom_bar(stat="identity") + theme(text=element_text(size=24))+
  geom_errorbar(aes(ymin=Mean-std, ymax=Mean+std),width=.2) + 
  theme_bw() + coord_flip() + theme(legend.position="none") + ylab("Average normalized MAE")+
  theme(axis.title.y=element_blank(),axis.text=element_text(color="black"),axis.ticks.y=element_blank())+
  scale_y_continuous(expand = c(0,0), limits=c(0,0.9), breaks = seq(0,0.8,0.2), labels =as.character(seq(0,0.8,0.2)))+
  scale_fill_manual(values=cbPalette)
print(p)
dev.off()
null device 
          1 
embed_fonts("mean-ranking2.pdf",outfile="mean-ranking2.pdf")
ggplotly(p)

dat_ = ddply(dat, .(method), summarize, Mean=mean(std))
foo = ddply(dat, .(method), summarize, Std=sd(std))
dat_ = merge(dat_,foo)
pdf(file="mean-ranking3.pdf",family="CM Roman",width=4,height=2)
p=ggplot(dat_, aes(x=reorder(method, -Mean), y=Mean, fill=method, group=method)) + 
  geom_bar(stat="identity") + theme(text=element_text(size=24))+
  geom_errorbar(aes(ymin=Mean-Std, ymax=Mean+Std),width=.2) + 
  theme_bw() + coord_flip() + theme(legend.position="none") + ylab("Average normalized standard deviation")+
  theme(axis.title.y=element_blank(),axis.text=element_text(color="black"),axis.ticks.y=element_blank())+
  scale_y_continuous(expand = c(0,0), limits=c(-0.02,0.45), breaks = seq(0,0.4,0.1), labels =as.character(seq(0,0.4,0.1)))+
  scale_fill_manual(values=cbPalette)
print(p)
dev.off()
null device 
          1 
embed_fonts("mean-ranking3.pdf",outfile="mean-ranking3.pdf")
ggplotly(p)

LS0tCnRpdGxlOiAiRmxvdyBhbmFseXNpcyBleHBsb3JhdGlvbiIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKYGBge3IgaW5jbHVkZT1GQUxTRX0KbGlicmFyeShwbG90bHkpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShwbHlyKQpsaWJyYXJ5KGV4dHJhZm9udCkKbGlicmFyeShncmlkRXh0cmEpCmZvbnRfaW5zdGFsbCgiZm9udGNtIikKbG9hZGZvbnRzKCkKYGBgCgoKIyMgRXZlbnQgbG9ncyBzdHVkaWVkOgoKYGBge3J9CmxvZ3MgPSByZWFkLmNzdigiLi4vbG9nc19zdW1tYXJ5LmNzdiIpCkRUOjpkYXRhdGFibGUobG9ncywgZmlsdGVyID0gInRvcCIpCmBgYAojIyBUcmFpbmluZyBjb25maWd1cmF0aW9uOgoKKiAyLzMgeCAxLzMgdGVtcG9yYWwgc3BsaXQKKiB6ZXJvIGJ1Y2tldGluZwoqIGFnZ3JlZ2F0ZSBlbmNvZGluZwoqIFhHYm9vc3Qgd2l0aCBoeXBlcnBhcmFtIG9wdGltaXphdGlvbiAobWFpbmx5LCAjIG9mIHRyZWVzKQoKCiMjIFN1cnZleWVkIHRlY2huaXF1ZXM6CgoqIHByZWRpY3RpdmUgZmxvdyBhbmFseXNpcyAoRkEpIC0gc2luZ2xlIGFuZCBtdWx0aXBsZSBtb2RlbCBvcHRpb25zICh6ZXJvIGJ1Y2tldGluZyBhbmQgcHJlZml4LWxlbmd0aCBidWNrZXRpbmcsIHJlc3BlY3RpdmVseSkKKiBhZGFwdGl2ZSBGQSAoYmFzZWQgb24gYSB2YXJpYWJsZSB0aHJlc2hvbGQgZS5nLiBtaW4gTl4yIHNhbXBsZXMgaW4gdGhlIHRyYWluaW5nIHNldCwgd2hlcmUgTiBpcyB0aGUgbnVtYmVyIG9mIGZlYXR1cmVzICkKKiBtZWFuIEZBIC0gZWFjaCBjeWNsZSB0aW1lIGFuZCBicmFuY2hpbmcgcHJvYmFiaWxpdHkgaXMgZGVyaXZlZCBiYXNlZCBvbiBoaXN0b3JpY2FsIGF2ZXJhZ2VzCgojIyBCYXNlbGluZXMKKiBzaW1wbGUgcmVncmVzc2lvbiAtIHJlbWFpbmluZyB0aW1lIGlzIHByZWRpY3RlZCBhcyBhIHNpbmdsZSB2YWx1ZSAoY29tcGFyYWJsZSB0byBbTWFzc2ltaWxpYW5vIGRlIExlb25pJ3MgYXBwcm9hY2hdKGh0dHA6Ly93d3cuc2NpZW5jZWRpcmVjdC5jb20vc2NpZW5jZS9hcnRpY2xlL3BpaS9TMDMwNjQzNzkxNTAwMTMxMykpCiogW1dpbF0oaHR0cHM6Ly93d3cucmVzZWFyY2hnYXRlLm5ldC9wdWJsaWNhdGlvbi8yMjA1MDM5NjFfVGltZV9wcmVkaWN0aW9uX2Jhc2VkX29uX3Byb2Nlc3NfbWluaW5nKQoqIFtBbmRyZWFzIFJvZ2dlLVNvbHRpXShodHRwczovL2xpbmsuc3ByaW5nZXIuY29tL2NoYXB0ZXIvMTAuMTAwNy85NzgtMy02NDItNDUwMDUtMV8yNykuCgoKYGBge3IgaW5jbHVkZT1GQUxTRX0KZGF0ID0gYygpCmZpbGVuYW1lcyA8LSBsaXN0LmZpbGVzKClbZ3JlcChwYXN0ZSgiXnZhbGlkYXRpb25fRkEyXyg/PS4qXFwuY3N2KSIsIHNlcD0nJyksIGxpc3QuZmlsZXMoKSwgcGVybD1UUlVFKV0KZm9yIChmaWxlbmFtZSBpbiBmaWxlbmFtZXMpIHsKICBpZihncmVwbCgiY29tYmluZWQiLCBmaWxlbmFtZSkgfCBncmVwbCgiaW5kZXgiLCBmaWxlbmFtZSkpIHsKICB0aGlzID0gcmVhZC5jc3YoZmlsZW5hbWUsIHNlcD0iLCIsIGhlYWRlciA9IFRSVUUpCiAgdGhpcyA9IHRoaXNbdGhpcyRucl9ldmVudHMgPCAxMSxdCiAgdGhpcyA9IHRoaXNbdGhpcyRtZXRyaWMgPT0gIm1hZSIsXQogIHRoaXMkbnJfY2FzZXMgPSB0aGlzJG5yX2Nhc2VzL3N1bSh0aGlzJG5yX2Nhc2VzKQogIHRoaXMkc2NvcmVfd2VpZ2h0ZWQgPSB0aGlzJHNjb3JlICogdGhpcyRucl9jYXNlcwogIHRoaXMkbWV0aG9kMiA9ICJGQSIKICB0aGlzJG1ldGhvZDMgPSBzdHJzcGxpdChmaWxlbmFtZSwiXyIpW1sxXV1bbGVuZ3RoKHN0cnNwbGl0KGZpbGVuYW1lLCJfIilbWzFdXSldCiAgdGhpcyRmaWxlbmFtZSA9IGZpbGVuYW1lCiAgZGF0ID0gcmJpbmQoZGF0LCB0aGlzKQogIH0KfQpwcmludChucm93KGRhdCkpCmBgYAoKYGBge3IgaW5jbHVkZT1GQUxTRX0KZmlsZW5hbWVzIDwtIGxpc3QuZmlsZXMoKVtncmVwKHBhc3RlKCJedmFsaWRhdGlvbl9kaXJlY3RfKD89LipcXC5jc3YpIiwgc2VwPScnKSwgbGlzdC5maWxlcygpLCBwZXJsPVRSVUUpXQpmb3IgKGZpbGVuYW1lIGluIGZpbGVuYW1lcykgewogIGlmKGdyZXBsKCJjb21iaW5lZCIsIGZpbGVuYW1lKSB8IGdyZXBsKCJpbmRleCIsIGZpbGVuYW1lKSkgewogIHRoaXMgPSByZWFkLmNzdihmaWxlbmFtZSwgc2VwPSIsIiwgaGVhZGVyID0gVFJVRSkKICB0aGlzID0gdGhpc1t0aGlzJG5yX2V2ZW50cyA8IDExLF0KICB0aGlzID0gdGhpc1t0aGlzJG1ldHJpYyA9PSAibWFlIixdCiAgdGhpcyRucl9jYXNlcyA9IHRoaXMkbnJfY2FzZXMvc3VtKHRoaXMkbnJfY2FzZXMpCiAgdGhpcyRzY29yZV93ZWlnaHRlZCA9IHRoaXMkc2NvcmUgKiB0aGlzJG5yX2Nhc2VzCiAgdGhpcyRtZXRob2QyID0gInJlZ3Jlc3Npb24iCiAgdGhpcyRtZXRob2QzID0gc3Ryc3BsaXQoZmlsZW5hbWUsIl8iKVtbMV1dW2xlbmd0aChzdHJzcGxpdChmaWxlbmFtZSwiXyIpW1sxXV0pXQogIHRoaXMkZmlsZW5hbWUgPSBmaWxlbmFtZQogIGRhdCA9IHJiaW5kKGRhdCwgdGhpcykKICB9Cn0KcHJpbnQobnJvdyhkYXQpKQpgYGAKCmBgYHtyfQpmaWxlbmFtZXMgPC0gbGlzdC5maWxlcygpW2dyZXAocGFzdGUoIl52YWxpZGF0aW9uX0FhbHN0Xyg/PS4qXFwuY3N2KSIsIHNlcD0nJyksIGxpc3QuZmlsZXMoKSwgcGVybD1UUlVFKV0KZm9yIChmaWxlbmFtZSBpbiBmaWxlbmFtZXMpIHsKICB0aGlzID0gcmVhZC5jc3YoZmlsZW5hbWUsIHNlcD0iLCIsIGhlYWRlciA9IFRSVUUpCiAgdGhpcyA9IHRoaXNbdGhpcyRucl9ldmVudHMgPCAxMSxdCiAgI3RoaXMgPSB0aGlzW3RoaXMkbWV0cmljID09ICJtYWUiLF0KICB0aGlzJG5yX2Nhc2VzID0gKHRoaXMkbnJfY2FzZXMvc3VtKHRoaXMkbnJfY2FzZXMpKQogIHRoaXMkc2NvcmVfd2VpZ2h0ZWQgPSB0aGlzJHNjb3JlICogdGhpcyRucl9jYXNlcwogIHRoaXMkbWV0aG9kMiA9IHRoaXMkY2xzCiAgdGhpcyRtZXRob2QzID0gIiIKICB0aGlzJGZpbGVuYW1lID0gZmlsZW5hbWUKICBkYXQgPSByYmluZChkYXQsIHRoaXMpCn0KYGBgCmBgYHtyfQpmaWxlbmFtZXMgPC0gbGlzdC5maWxlcygpW2dyZXAocGFzdGUoIl52YWxpZGF0aW9uX1NQTl8oPz0uKlxcLmNzdikiLCBzZXA9JycpLCBsaXN0LmZpbGVzKCksIHBlcmw9VFJVRSldCmZvciAoZmlsZW5hbWUgaW4gZmlsZW5hbWVzKSB7CiAgdGhpcyA9IHJlYWQuY3N2KGZpbGVuYW1lLCBzZXA9IiwiLCBoZWFkZXIgPSBUUlVFKQogIHRoaXMgPSB0aGlzW3RoaXMkbnJfZXZlbnRzIDwgMTEsXQogICN0aGlzID0gdGhpc1t0aGlzJG1ldHJpYyA9PSAibWFlIixdCiAgdGhpcyRucl9jYXNlcyA9IHRoaXMkbnJfY2FzZXMvc3VtKHRoaXMkbnJfY2FzZXMpCiAgdGhpcyRzY29yZV93ZWlnaHRlZCA9IHRoaXMkc2NvcmUgKiB0aGlzJG5yX2Nhc2VzCiAgdGhpcyRtZXRob2QyID0gdGhpcyRjbHMKICB0aGlzJG1ldGhvZDMgPSAiIgogIHRoaXMkZmlsZW5hbWUgPSBmaWxlbmFtZQogIGRhdCA9IHJiaW5kKGRhdCwgdGhpcykKfQpgYGAKCgpgYGB7ciBldmFsPUZBTFNFLCBpbmNsdWRlPUZBTFNFfQpzaXplMSA9IG5yb3coZGF0KQpkYXQgPSBkYXRbY29tcGxldGUuY2FzZXMoZGF0KSxdCmRhdCRkYXRhc2V0ID0gZHJvcGxldmVscyhkYXQkZGF0YXNldCkKc2l6ZTIgPSBucm93KGRhdCkKaWYoKHNpemUxIC0gc2l6ZTIpL3NpemUyID4gMC4wMSkgIHsKICBwcmludCgiV2FybmluZyEgTW9yZSB0aGFuIDElIG9mIGluY29tcGxldGUgcm93cyIpCn0KYGBgCgpgYGB7ciBpbmNsdWRlPUZBTFNFfQpkYXQkbWV0aG9kM1tkYXQkbWV0aG9kMyA9PSAiMzAwMDAwMC5jc3YiXSA9ICJtZWFuIgpkYXQkbWV0aG9kM1tkYXQkbWV0aG9kMyA9PSAiMzAuY3N2IiAmIGRhdCRtZXRob2QyID09ICJGQSJdID0gInByZWRpY3RpdmUiCmRhdCRtZXRob2QzW2RhdCRtZXRob2QzID09ICIzMC5jc3YiICYgZGF0JG1ldGhvZDIgPT0gInJlZ3Jlc3Npb24iXSA9ICJzaW1wbGUiCiMgZGF0JG1ldGhvZDNbZGF0JG1ldGhvZDMgPT0gIjMwLmNzdiIgJiBkYXQkbWV0aG9kMiA9PSAiaXNvdG9uaWNfRkEiXSA9ICJpc290b25pYyIKIyBkYXQkbWV0aG9kM1tkYXQkbWV0aG9kMyA9PSAiMzAuY3N2IiAmIGRhdCRtZXRob2QyID09ICJzaWdtb2lkX0ZBIl0gPSAic2lnbW9pZCIKZGF0JG1ldGhvZDNbZGF0JG1ldGhvZDMgPT0gIjUuY3N2Il0gPSAiYWRhcHRpdmUiCmRhdCRtZXRob2QzW2RhdCRtZXRob2QzID09ICIxMC5jc3YiXSA9ICJhZGFwdGl2ZSIKZGF0JG1ldGhvZDNbZGF0JG1ldGhvZDMgPT0gIjIwLmNzdiJdID0gImFkYXB0aXZlIgpkYXQkbWV0aG9kM1tkYXQkbWV0aG9kMyA9PSAic3FydC5jc3YiXSA9ICJhZGFwdGl2ZSIKCmRhdCRtZXRob2QgPSBwYXN0ZShkYXQkbWV0aG9kMywgZGF0JG1ldGhvZDIsIGRhdCRtZXRob2QsIHNlcCA9ICJfIikKYGBgCgoKYGBge3IgaW5jbHVkZT1GQUxTRX0KZGF0JHNjb3JlID0gZGF0JHNjb3JlIC8gKDM2MDAqMjQpICMgbWFrZSBkYXlzCmRhdCRzY29yZV93ZWlnaHRlZCA9IGRhdCRzY29yZV93ZWlnaHRlZCAvICgzNjAwKjI0KQpgYGAKCiMjIEF2ZXJhZ2UgTUFFIGFjcm9zcyBhbGwgcHJlZml4IGxlbmd0aHMgZm9yIGFsbCBkYXRhc2V0cyBhbmQgbWV0aG9kczoKYGBge3J9CmFnZ19NQUVfdW53ID0gYXMuZGF0YS5mcmFtZShhZ2dyZWdhdGUoZGF0JHNjb3JlLGJ5PWxpc3QoZGF0YXNldD1kYXQkZGF0YXNldCwgbWV0aG9kPWRhdCRtZXRob2QpLCBtZWFuKSkKYWdnX01BRV93ID0gYXMuZGF0YS5mcmFtZShhZ2dyZWdhdGUoZGF0JHNjb3JlX3dlaWdodGVkLGJ5PWxpc3QoZGF0YXNldD1kYXQkZGF0YXNldCwgbWV0aG9kPWRhdCRtZXRob2QpLCBzdW0pKQphZ2dfTUFFX3Vud1ssM10gPSByb3VuZChhZ2dfTUFFX3Vud1ssM10sIDMpCmFnZ19NQUVfd1ssM10gPSByb3VuZChhZ2dfTUFFX3dbLDNdLCAzKQpjb2xuYW1lcyhhZ2dfTUFFX3VudylbM10gPSAiQXZnX3Vud2VpZ2h0ZWRfTUFFX2RheXMiCmNvbG5hbWVzKGFnZ19NQUVfdylbM10gPSAiQXZnX3dlaWdodGVkX01BRV9kYXlzIgphZ2dfTUFFID0gbWVyZ2UoYWdnX01BRV93LCBhZ2dfTUFFX3VudykKYWdnX01BRSA9IGFnZ19NQUVbd2l0aChhZ2dfTUFFLCBvcmRlcihkYXRhc2V0LCBBdmdfd2VpZ2h0ZWRfTUFFX2RheXMsIG1ldGhvZCA9ICJyYWRpeCIpKSwgXQoKRFQ6OmRhdGF0YWJsZShhZ2dfTUFFLCBmaWx0ZXIgPSAidG9wIikKYGBgCgpgYGB7cn0KdG1wID0gZGRwbHkoZGF0LCAuKGRhdGFzZXQsbWV0aG9kKSwgc3VtbWFyaXplLCBzdGQ9c2Qoc2NvcmUpKQp0bXAkc3RkID0gcm91bmQodG1wJHN0ZCwzKQphZ2dfTUFFID0gbWVyZ2UoYWdnX01BRSwgdG1wKQphZ2dfTUFFJEF2Z191bndlaWdodGVkX01BRV9kYXlzPU5VTEwKYWdnX01BRSA9IGFnZ19NQUVbd2l0aChhZ2dfTUFFLCBvcmRlcihkYXRhc2V0LCBBdmdfd2VpZ2h0ZWRfTUFFX2RheXMsIG1ldGhvZCA9ICJyYWRpeCIpKSwgXQpEVDo6ZGF0YXRhYmxlKGFnZ19NQUUsIGZpbHRlciA9ICJ0b3AiKQpgYGAKCiMjIFBsb3RzIE1BRSB2cyBwcmVmaXggc2l6ZQpgYGB7cn0KZGF0JG1ldGhvZFtkYXQkbWV0aG9kPT0iX2Jlc3RfQWFsc3QiXSA9ICJUUyAoYmVzdCkiCmRhdCRtZXRob2RbZGF0JG1ldGhvZD09ImFkYXB0aXZlX0ZBX3plcm9fY29tYmluZWQiXSA9ICJhZGFwdGl2ZSBGQSIKZGF0JG1ldGhvZFtkYXQkbWV0aG9kPT0ibWVhbl9GQV96ZXJvX2NvbWJpbmVkIl0gPSAibWVhbiBGQSIKZGF0JG1ldGhvZFtkYXQkbWV0aG9kPT0icHJlZGljdGl2ZV9GQV9wcmVmaXhfaW5kZXgiXSA9ICJwcmVkaWN0aXZlIEZBIChtdWx0aXBsZSkiCmRhdCRtZXRob2RbZGF0JG1ldGhvZD09InByZWRpY3RpdmVfRkFfemVyb19jb21iaW5lZCJdID0gInByZWRpY3RpdmUgRkEgKHNpbmdsZSkiCmRhdCRtZXRob2RbZGF0JG1ldGhvZD09InNpbXBsZV9yZWdyZXNzaW9uX3ByZWZpeF9pbmRleCJdID0gImJsYWNrIGJveCAobXVsdGlwbGUpIgpkYXQkbWV0aG9kW2RhdCRtZXRob2Q9PSJzaW1wbGVfcmVncmVzc2lvbl96ZXJvX2NvbWJpbmVkIl0gPSAiYmxhY2sgYm94IChzaW5nbGUpIgpkYXQkbWV0aG9kW2RhdCRtZXRob2Q9PSJfU1BOX1JvZ2dlLVNvbHRpIl0gPSAic3RvY2hhc3RpYyBQZXRyaSBuZXQiCgpsZXZlbHMoZGF0JGRhdGFzZXQpWzFdID0gIkJQSUMnMTIgQSIKbGV2ZWxzKGRhdCRkYXRhc2V0KVsyXSA9ICJCUElDJzEyIE8iCmxldmVscyhkYXQkZGF0YXNldClbM10gPSAiQlBJQycxMiBXX258MXwiCmxldmVscyhkYXQkZGF0YXNldClbNF0gPSAiQlBJQycxMiBXIgpsZXZlbHMoZGF0JGRhdGFzZXQpWzVdID0gIkNSIgpsZXZlbHMoZGF0JGRhdGFzZXQpWzZdID0gIkhlbHBkZXNrIgpsZXZlbHMoZGF0JGRhdGFzZXQpWzddID0gIkhvc3BpdGFsIgpsZXZlbHMoZGF0JGRhdGFzZXQpWzhdID0gIkludm9pY2UiCmxldmVscyhkYXQkZGF0YXNldClbOV0gPSAiUlRGTVAiCgpmb3IgKGRmIGluIGxldmVscyhkYXQkZGF0YXNldCkpIHsKICBwID0gZ2dwbG90KGRhdFtkYXQkZGF0YXNldCA9PSBkZixdLCBhZXMoeCA9IG5yX2V2ZW50cywgeSA9IHNjb3JlLCBjb2xvciA9IG1ldGhvZCkpICsgZ2VvbV9wb2ludCgpICsgZ2VvbV9saW5lKCkgKyAKICAgICNmYWNldF93cmFwKH4gZGF0YXNldCwgbmNvbCA9IDMsIHNjYWxlcyA9ICJmcmVlIikgKyAKICAgIHlsYWIoIk1BRSwgZGF5cyIpICsgeGxhYigiUHJlZml4IGxlbmd0aCIpKwogICAgZ2d0aXRsZShkZikKICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbj0idG9wIikrCiAgICB0aGVtZShwYW5lbC5iYWNrZ3JvdW5kID0gZWxlbWVudF9yZWN0KGZpbGwgPSAnd2hpdGUnLCBjb2xvdXIgPSAnYmxhY2snLHNpemU9MC41KSkgKyAKICAgIHRoZW1lKHBhbmVsLmdyaWQubWFqb3IgPSBlbGVtZW50X2xpbmUoY29sb3VyID0gJ2xpZ2h0Z3JleScsIHNpemUgPSAwLjMpKSsKICB0aGVtZShsZWdlbmQuYmFja2dyb3VuZCA9IGVsZW1lbnRfcmVjdChjb2xvdXIgPSAnd2hpdGUnLHNpemUgPSAwLjEsIGxpbmV0eXBlPSdzb2xpZCcpKSsKICB0aGVtZShsZWdlbmQudGl0bGU9ZWxlbWVudF9ibGFuaygpKQogICNwcmludChwKQp9CmBgYAoKIyMgSW50ZXJhY3RpdmUgcGxvdHMgY2FuIGJlIGFjY2Vzc2VkIFtoZXJlXShodHRwczovL3Bsb3QubHkvfnZlcmVuaWNoLzkzLykKYGBge3IgaW5jbHVkZT1GQUxTRX0KY2JQYWxldHRlIDwtIGMoIiM5OTk5OTkiLCAiI0U2OUYwMCIsICIjNTZCNEU5IiwgIiMwMDlFNzMiLCAiI0YwRTQ0MiIsICIjMDA3MkIyIiwgIiNENTVFMDAiLCAiI0NDNzlBNyIpCmNiYlBhbGV0dGUgPC0gYygiIzAwMDAwMCIsICIjRTY5RjAwIiwgIiM1NkI0RTkiLCAiIzAwOUU3MyIsICIjRjBFNDQyIiwgIiMwMDcyQjIiLCAiI0Q1NUUwMCIsICIjQ0M3OUE3IikKcCA9IGdncGxvdChkYXQsIGFlcyh4ID0gbnJfZXZlbnRzLCB5ID0gc2NvcmUsIGNvbG9yID0gbWV0aG9kKSkgKyBnZW9tX3BvaW50KHNpemU9MC41KSArIGdlb21fbGluZShzaXplPTAuNCkgKyAKICBmYWNldF93cmFwKH4gZGF0YXNldCwgbmNvbCA9IDMsIHNjYWxlcyA9ICJmcmVlIikgKyB5bGFiKCJNQUUsIGRheXMiKSArIHhsYWIoIlByZWZpeCBsZW5ndGgiKSsKICB0aGVtZShsZWdlbmQucG9zaXRpb249InRvcCIpKwogICAgdGhlbWUocGFuZWwuYmFja2dyb3VuZCA9IGVsZW1lbnRfcmVjdChmaWxsID0gJ3doaXRlJywgY29sb3VyID0gJ2JsYWNrJyxzaXplPTAuNSkpICsgCiAgICB0aGVtZShwYW5lbC5ncmlkLm1ham9yID0gZWxlbWVudF9saW5lKGNvbG91ciA9ICdsaWdodGdyZXknLCBzaXplID0gMC4zKSkrCiAgdGhlbWUobGVnZW5kLmJhY2tncm91bmQgPSBlbGVtZW50X3JlY3QoY29sb3VyID0gJ3doaXRlJyxzaXplID0gMC4xLCBsaW5ldHlwZT0nc29saWQnKSkrCiAgdGhlbWUobGVnZW5kLnRpdGxlPWVsZW1lbnRfYmxhbmsoKSkgKwogIHRoZW1lKHRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTApKSsKICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KGNvbG91cj0iYmxhY2siKSkrCiAgI3NjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygiYmxhY2siLCJyZWQiLCJibHVlIiwiZGFya2dyZWVuIiwib3JhbmdlIiwiZGFya2JsdWUiLCJ2aW9sZXQiLCJncmF5IikpKwogIHNjYWxlX2NvbG91cl9tYW51YWwodmFsdWVzPWNiUGFsZXR0ZSkKYXBpX2NyZWF0ZShwLCAiZmxvdy1hbmFseXNpcy1NQUUtYWxsIiwgc2hhcmluZyA9ICJwdWJsaWMiKQpgYGAKCmBgYHtyIGluY2x1ZGU9RkFMU0V9CnBkZihmaWxlPSJNQUUtYWxsLnBkZiIsZmFtaWx5PSJDTSBSb21hbiIsd2lkdGg9NixoZWlnaHQ9NikKcHJpbnQocCkKZGV2Lm9mZigpCmVtYmVkX2ZvbnRzKCJNQUUtYWxsLnBkZiIsb3V0ZmlsZT0iTUFFLWFsbC5wZGYiKQpgYGAKCiMjIyBmb3IgcGFwZXIgb25seSAjIyMKCmBgYHtyfQphZ2dfTUFFJG1ldGhvZFthZ2dfTUFFJG1ldGhvZD09Il9iZXN0X0FhbHN0Il0gPSAiVFMgKGJlc3QpIgphZ2dfTUFFJG1ldGhvZFthZ2dfTUFFJG1ldGhvZD09ImFkYXB0aXZlX0ZBX3plcm9fY29tYmluZWQiXSA9ICJhZGFwdGl2ZSBGQSIKYWdnX01BRSRtZXRob2RbYWdnX01BRSRtZXRob2Q9PSJtZWFuX0ZBX3plcm9fY29tYmluZWQiXSA9ICJtZWFuIEZBIgphZ2dfTUFFJG1ldGhvZFthZ2dfTUFFJG1ldGhvZD09InByZWRpY3RpdmVfRkFfcHJlZml4X2luZGV4Il0gPSAicHJlZGljdGl2ZSBGQSAobXVsdGlwbGUpIgphZ2dfTUFFJG1ldGhvZFthZ2dfTUFFJG1ldGhvZD09InByZWRpY3RpdmVfRkFfemVyb19jb21iaW5lZCJdID0gInByZWRpY3RpdmUgRkEgKHNpbmdsZSkiCmFnZ19NQUUkbWV0aG9kW2FnZ19NQUUkbWV0aG9kPT0ic2ltcGxlX3JlZ3Jlc3Npb25fcHJlZml4X2luZGV4Il0gPSAiYmxhY2sgYm94IChtdWx0aXBsZSkiCmFnZ19NQUUkbWV0aG9kW2FnZ19NQUUkbWV0aG9kPT0ic2ltcGxlX3JlZ3Jlc3Npb25femVyb19jb21iaW5lZCJdID0gImJsYWNrIGJveCAoc2luZ2xlKSIKYWdnX01BRSRtZXRob2RbYWdnX01BRSRtZXRob2Q9PSJfU1BOX1JvZ2dlLVNvbHRpIl0gPSAic3RvY2hhc3RpYyBQZXRyaSBuZXQiCmRhdCA9IGRhdGEuZnJhbWUoKQoKZm9yIChkc2V0IGluIGxldmVscyhhZ2dfTUFFJGRhdGFzZXQpKSB7CiAgZGYgPSBhZ2dfTUFFW2FnZ19NQUUkZGF0YXNldD09ZHNldCxdCiAgZGYkT3JkZXIgPSByYW5rKGRmJEF2Z193ZWlnaHRlZF9NQUVfZGF5cyx0aWVzLm1ldGhvZCA9ICJtaW4iKQogIGRhdCA9IHJiaW5kKGRhdCwgZGYpCn0KCmRhdF8gPSBkZHBseShkYXQsIC4obWV0aG9kKSwgc3VtbWFyaXplLCBNZWFuPW1lYW4oT3JkZXIpKQpmb28gPSBkZHBseShkYXQsIC4obWV0aG9kKSwgc3VtbWFyaXplLCBzdGQ9c2QoT3JkZXIpKQpkYXRfID0gbWVyZ2UoZGF0Xyxmb28pCmRhdF8gPSBkYXRfW29yZGVyKGRhdF8kTWVhbiksXQpgYGAKCmBgYHtyfQpwZGYoZmlsZT0ibWVhbi1yYW5raW5nLnBkZiIsZmFtaWx5PSJDTSBSb21hbiIsd2lkdGg9NCxoZWlnaHQ9MikKcD1nZ3Bsb3QoZGF0XywgYWVzKHg9cmVvcmRlcihtZXRob2QsIC1NZWFuKSwgeT1NZWFuLCBmaWxsPW1ldGhvZCwgZ3JvdXA9bWV0aG9kKSkgKyAKICBnZW9tX2JhcihzdGF0PSJpZGVudGl0eSIpICsgdGhlbWUodGV4dD1lbGVtZW50X3RleHQoc2l6ZT0yNCkpKwogIGdlb21fZXJyb3JiYXIoYWVzKHltaW49TWVhbi1zdGQsIHltYXg9TWVhbitzdGQpLHdpZHRoPS4yKSArIAogIHRoZW1lX2J3KCkgKyBjb29yZF9mbGlwKCkgKyB0aGVtZShsZWdlbmQucG9zaXRpb249Im5vbmUiKSArIHlsYWIoIm1lYW4gcmFua2luZyIpKwogIHRoZW1lKGF4aXMudGl0bGUueT1lbGVtZW50X2JsYW5rKCksYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChjb2xvcj0iYmxhY2siKSxheGlzLnRpY2tzLnk9ZWxlbWVudF9ibGFuaygpKSsKICBzY2FsZV95X2NvbnRpbnVvdXMoZXhwYW5kID0gYygwLDApLCBsaW1pdHM9YygwLDkpLGJyZWFrcyA9IHNlcSgxLDcsMiksIGxhYmVscyA9YXMuY2hhcmFjdGVyKHNlcSgxLDcsMikpKSsKICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXM9Y2JQYWxldHRlKQpwcmludChwKQpkZXYub2ZmKCkKZW1iZWRfZm9udHMoIm1lYW4tcmFua2luZy5wZGYiLG91dGZpbGU9Im1lYW4tcmFua2luZy5wZGYiKQpnZ3Bsb3RseShwKQpgYGAKYGBge3J9CmRhdCRjYXNlX2R1cmF0aW9uID0gMApkYXQkY2FzZV9kdXJhdGlvbltkYXQkZGF0YXNldD09IkJQSTIwMTJBIl0gPSA3LjUKZGF0JGNhc2VfZHVyYXRpb25bZGF0JGRhdGFzZXQ9PSJCUEkyMDEyTyJdID0gMTUuMQpkYXQkY2FzZV9kdXJhdGlvbltkYXQkZGF0YXNldD09IkJQSTIwMTJXIl0gPSAxMS40CmRhdCRjYXNlX2R1cmF0aW9uW2RhdCRkYXRhc2V0PT0iQlBJMjAxMldfbm9fZHVwIl0gPSAxMS40CmRhdCRjYXNlX2R1cmF0aW9uW2RhdCRkYXRhc2V0PT0iQ3JlZGl0UmVxdWlyZW1lbnQiXSA9IDAuOTUKZGF0JGNhc2VfZHVyYXRpb25bZGF0JGRhdGFzZXQ9PSJoZWxwZGVzayJdID0gNy4zCmRhdCRjYXNlX2R1cmF0aW9uW2RhdCRkYXRhc2V0PT0ibWluaXRfaW52b2ljZV8xMCJdID0gMi4yMjUKZGF0JGNhc2VfZHVyYXRpb25bZGF0JGRhdGFzZXQ9PSJ0cmFmZmljX2ZpbmVzXzEzOSJdID0gNTgyCmRhdCRjYXNlX2R1cmF0aW9uW2RhdCRkYXRhc2V0PT0iaG9zcGl0YWxfYmlsbGluZ185NzciXSA9IDE2NS4yCgpkYXQkQXZnX3dlaWdodGVkX01BRV9kYXlzID0gZGF0JEF2Z193ZWlnaHRlZF9NQUVfZGF5cyAvIGRhdCRjYXNlX2R1cmF0aW9uCmRhdCRzdGQgPSBkYXQkc3RkIC8gZGF0JGNhc2VfZHVyYXRpb24KZGF0JE9yZGVyID0gTlVMTApgYGAKCgpgYGB7cn0KZGF0XyA9IGRkcGx5KGRhdCwgLihtZXRob2QpLCBzdW1tYXJpemUsIE1lYW49bWVhbihBdmdfd2VpZ2h0ZWRfTUFFX2RheXMpKQpmb28gPSBkZHBseShkYXQsIC4obWV0aG9kKSwgc3VtbWFyaXplLCBzdGQ9c2QoQXZnX3dlaWdodGVkX01BRV9kYXlzKSkKZGF0XyA9IG1lcmdlKGRhdF8sZm9vKQpwZGYoZmlsZT0ibWVhbi1yYW5raW5nMi5wZGYiLGZhbWlseT0iQ00gUm9tYW4iLHdpZHRoPTQsaGVpZ2h0PTIpCnA9Z2dwbG90KGRhdF8sIGFlcyh4PXJlb3JkZXIobWV0aG9kLCAtTWVhbiksIHk9TWVhbiwgZmlsbD1tZXRob2QsIGdyb3VwPW1ldGhvZCkpICsgCiAgZ2VvbV9iYXIoc3RhdD0iaWRlbnRpdHkiKSArIHRoZW1lKHRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MjQpKSsKICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPU1lYW4tc3RkLCB5bWF4PU1lYW4rc3RkKSx3aWR0aD0uMikgKyAKICB0aGVtZV9idygpICsgY29vcmRfZmxpcCgpICsgdGhlbWUobGVnZW5kLnBvc2l0aW9uPSJub25lIikgKyB5bGFiKCJBdmVyYWdlIG5vcm1hbGl6ZWQgTUFFIikrCiAgdGhlbWUoYXhpcy50aXRsZS55PWVsZW1lbnRfYmxhbmsoKSxheGlzLnRleHQ9ZWxlbWVudF90ZXh0KGNvbG9yPSJibGFjayIpLGF4aXMudGlja3MueT1lbGVtZW50X2JsYW5rKCkpKwogIHNjYWxlX3lfY29udGludW91cyhleHBhbmQgPSBjKDAsMCksIGxpbWl0cz1jKDAsMC45KSwgYnJlYWtzID0gc2VxKDAsMC44LDAuMiksIGxhYmVscyA9YXMuY2hhcmFjdGVyKHNlcSgwLDAuOCwwLjIpKSkrCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzPWNiUGFsZXR0ZSkKcHJpbnQocCkKZGV2Lm9mZigpCmVtYmVkX2ZvbnRzKCJtZWFuLXJhbmtpbmcyLnBkZiIsb3V0ZmlsZT0ibWVhbi1yYW5raW5nMi5wZGYiKQpnZ3Bsb3RseShwKQpgYGAKYGBge3J9CmRhdF8gPSBkZHBseShkYXQsIC4obWV0aG9kKSwgc3VtbWFyaXplLCBNZWFuPW1lYW4oc3RkKSkKZm9vID0gZGRwbHkoZGF0LCAuKG1ldGhvZCksIHN1bW1hcml6ZSwgU3RkPXNkKHN0ZCkpCmRhdF8gPSBtZXJnZShkYXRfLGZvbykKcGRmKGZpbGU9Im1lYW4tcmFua2luZzMucGRmIixmYW1pbHk9IkNNIFJvbWFuIix3aWR0aD00LGhlaWdodD0yKQpwPWdncGxvdChkYXRfLCBhZXMoeD1yZW9yZGVyKG1ldGhvZCwgLU1lYW4pLCB5PU1lYW4sIGZpbGw9bWV0aG9kLCBncm91cD1tZXRob2QpKSArIAogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5IikgKyB0aGVtZSh0ZXh0PWVsZW1lbnRfdGV4dChzaXplPTI0KSkrCiAgZ2VvbV9lcnJvcmJhcihhZXMoeW1pbj1NZWFuLVN0ZCwgeW1heD1NZWFuK1N0ZCksd2lkdGg9LjIpICsgCiAgdGhlbWVfYncoKSArIGNvb3JkX2ZsaXAoKSArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbj0ibm9uZSIpICsgeWxhYigiQXZlcmFnZSBub3JtYWxpemVkIHN0YW5kYXJkIGRldmlhdGlvbiIpKwogIHRoZW1lKGF4aXMudGl0bGUueT1lbGVtZW50X2JsYW5rKCksYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChjb2xvcj0iYmxhY2siKSxheGlzLnRpY2tzLnk9ZWxlbWVudF9ibGFuaygpKSsKICBzY2FsZV95X2NvbnRpbnVvdXMoZXhwYW5kID0gYygwLDApLCBsaW1pdHM9YygtMC4wMiwwLjQ1KSwgYnJlYWtzID0gc2VxKDAsMC40LDAuMSksIGxhYmVscyA9YXMuY2hhcmFjdGVyKHNlcSgwLDAuNCwwLjEpKSkrCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzPWNiUGFsZXR0ZSkKcHJpbnQocCkKZGV2Lm9mZigpCmVtYmVkX2ZvbnRzKCJtZWFuLXJhbmtpbmczLnBkZiIsb3V0ZmlsZT0ibWVhbi1yYW5raW5nMy5wZGYiKQpnZ3Bsb3RseShwKQpgYGAKCg==